import scrapy
from kickstart import items
import json
import re
import copy
import pickle
from  bson.binary import Binary

class CtripSpider(scrapy.Spider):
    name = "ctriphotel"
    allowed_domains = ["hotels.ctrip.com", "images4.c-ctrip.com"]
    start_urls = ["http://hotels.ctrip.com/Domestic/Tool/AjaxGetCitySuggestion.aspx"]
    def __init__(self, spiderid=None, step=None, cityfilepath=None, *args, **kwargs):
        super(CtripSpider, self).__init__(*args, **kwargs)
        if spiderid and step:
            self.start = int(spiderid) * int(step)
            self.end = self.start + int(step)
        else: 
            self.start = None
            self.end = None
        print self.start, self.end
        
        self.cities = None
        if cityfilepath:
            cityfile = open(cityfilepath, 'rb')
            cities = pickle.load(cityfile)
            #relay on top level script control
            if self.start >= 0:
                if self.end and self.end <= len(cities):
                    self.cities = cities[self.start:self.end]
                else:
                    self.cities = cities[self.start:]

    def createCity(self, cname, data, group):
        city = items.CtripCity()
        tmps = data.split(':')[1].split('|')
        city['cname'] = cname
        city['pinyin'] = tmps[0].replace('"', '').lower()
        city['cid'] = tmps[2].replace('"', '')
        city['group'] = group.split(':')[1]
        return city

    def json_parse(self, data):
	replace_fmt = re.compile('\{|\}')
	replace_fmt1 = re.compile('\[|\]')	
	json_strs_tmp = replace_fmt.sub('', data)
	json_strs = replace_fmt1.sub('', json_strs_tmp)
        datas = json_strs.decode('gb2312').split(",")
	index = 0
	cities = []
	while index < len(datas):
            city = None
            tmps = datas[index].split(':')
            if tmps[0] == 'display':
                city = self.createCity(tmps[1], datas[index+1], datas[index+2])
            elif tmps[1] == 'display':
                city = self.createCity(tmps[2], datas[index+1], datas[index+2])
            if city:
                cities.append(city)
            index += 3
	return cities
			
    #def storeHTMLCity(self, cid, response):

    def storePic(self, response):
        hotelPic = items.CtripHotelPIC()
        hotelPic['url'] = response.url
        hotelPic['pic'] = Binary(response.body)#.decode('ascii').encode('utf-8')
        hotelPic['hotelid'] = response.meta['hotelid']
        yield hotelPic

    def parseHotelCommentsPage(self, response): 
        commentHTML = items.CtripHotelCommentHTML()
        commentHTML['hotelid'] = response.meta['hotelid']
        commentHTML['url'] = response.url
        commentHTML['html'] = response.body.decode("gb18030").encode("UTF-8")
        commentHTML['pageid'] = response.meta['cpageid']
        yield commentHTML

        #deal page one:
        comments = response.selector.xpath('//div[@class="comment_detail_list"]/div[@class="comment_block"]')
        for comm in comments:
            comment = items.CtripHotelComment()
            comment['hotelid'] = response.meta['hotelid']
            comment['date'] = comm.xpath('span[@class="date"]/a/text()').extract()
            comment['score'] = comm.xpath('span[@class="score"]/span[@class="n"]/text()').extract()
            comment['uname'] = comm.xpath('div[@class="user_info"]/p[@class="name"]/span/text()').extract()
            comment['comment'] = comm.xpath('div[@class="comment_txt"]/p[@class="J_commentDetail"]/text()').extract()
            yield comment
        
    def parseHotelComments(self, response):
        response.meta['cpageid'] = 1
        commentHTML = items.CtripHotelCommentHTML()
        commentHTML['hotelid'] = response.meta['hotelid']
        commentHTML['url'] = response.url
        commentHTML['html'] = response.body.decode("gb18030").encode("UTF-8")
        commentHTML['pageid'] = response.meta['cpageid']
        yield commentHTML
        print "comment html", response.url
        #deal page one:
        #comments = response.selector.xpath('//div[@class="comment_tab_main"]/div[@class="comment_detail_list"]/div[@class="comment_block"]')
        comments = response.selector.xpath('//*[@id="commentList"]/div[1]/div/div/div[3]/div[@class="comment_block"]')
        for comm in comments:
            comment = items.CtripHotelComment()
            comment['hotelid'] = response.meta['hotelid']
            comment['date'] = comm.xpath('div[@class="comment_main"]/p[@class="comment_title"]/span[@class="date"]/a/text()').extract()
            comment['score'] = comm.xpath('div[@class="comment_main"]/p/span[@class="score"]/span[@class="n"]/text()').extract()
            comment['uname'] = comm.xpath('div[@class="user_info"]/p[@class="name"]/span/text()').extract()
            comment['comment'] = comm.xpath('div[@class="comment_main"]/div[@class="comment_txt"]/p[@class="J_commentDetail"]/text()').extract()
            yield comment
        #deal other pages
        pages = response.selector.xpath('//input[@id="cTotalPageNum"]/@value').extract()
        pagenum = 0
        if len(pages) > 0:
            pagenum = int(pages[0])
        #if pagenum > 5:
         #   pagenum = 5

        prefix = "http://hotels.ctrip.com/hotel/dianping/%s_p%dt0.html"
        for page in range(2, pagenum+1):
            tmp_url = prefix % (response.meta['hotelid'], page)
            response.meta['cpageid'] = page
            yield scrapy.Request(tmp_url, callback=self.parseHotelCommentsPage, meta=response.meta)
            print "request url ", tmp_url
    
    #TODO store response to disk use item.
    def parseHotelInfo(self, response):
        hotel = items.CtripHotel()
        hotel['id'] = response.meta['hotelid']
        hotel['cid'] = response.meta['cid']
        hotel['cname'] = response.selector.xpath('//h2[@class="cn_n"]/text()').extract()
        hotel['ename'] = response.selector.xpath('//h2[@class="en_n"]/text()').extract()
        loc = response.selector.xpath('//span[@id="ctl00_MainContentPlaceHolder_commonHead1_lnkLocation"]/text()').extract()
        lcity = response.selector.xpath('//span[@id="ctl00_MainContentPlaceHolder_commonHead1_lnkCity"]/text()').extract()
        laddress = response.selector.xpath('//span[@id="ctl00_MainContentPlaceHolder_commonHead1_lbAddress"]/text()').extract()
        lroad = response.selector.xpath('//span[@id="ctl00_MainContentPlaceHolder_commonHead1_lnkRoadCross"]/text()').extract()
        addr = ""
        if len(loc) > 0:
            addr += loc[0]
        if len(lcity) > 0:
            addr += " " + lcity[0]
        if len(laddress) > 0:
            addr += " " + laddress[0]
        if len(lroad) > 0:
            addr += " " + lroad[0]
        hotel['address'] = addr
        #hotel['address'] = " ".join([loc[0], lcity[0], laddress[0], lroad[0]])
        zonestr = response.selector.xpath('//a[@id="ctl00_MainContentPlaceHolder_commonHead1_lnkMapZone"]/@href').extract()
        index = -1
        if len(zonestr) > 0:
            index = zonestr[0].rfind('zone')
        if  index > 0:
            hotel['zid'] = zonestr[0][index+4:]
        else:
            hotel['zid'] = -1

        hotel['score'] = response.selector.xpath('//a[@id="LinkReview2"]/span[@class="score"]/text()').extract()
        
        default_picurl = response.selector.xpath('//meta[@itemprop="image"]/@content').extract()        
        if len(default_picurl) > 0:
            print "url: ",  default_picurl
            yield scrapy.Request(default_picurl[0], callback=self.storePic, meta={'hotelid': response.meta['hotelid']})
        
        yield hotel
        print "zone", zonestr
        
        print hotel

        hotelHTML = items.CtripHotelHTML()
        hotelHTML['cid'] = response.meta['cid']
        hotelHTML['hotelid'] = response.meta['hotelid']
        hotelHTML['url'] = response.url
        hotelHTML['html'] = response.body.decode("gb18030").encode("UTF-8")
        yield hotelHTML

        comment_num = response.selector.xpath('//span[@class="commnet_num"]/span[@itemprop="reviewCount"]/text()').extract()
        print "comment", comment_num
        if len(comment_num) <= 0:
            print "comment_num < 0", response.url, comment_num
            return
        prefix = "http://hotels.ctrip.com%s" 
        comments_url = response.selector.xpath('//a[@id="id_comment_view"]/@href').extract()
        if  len(comments_url) > 0:
            tmp_url = prefix % (comments_url[0])
            yield scrapy.Request(tmp_url, callback=self.parseHotelComments, meta=response.meta)

    def parseHotelsPage(self, response):
        cityHTML = items.CtripCityHTML()
        cityHTML['cid'] = response.meta['cid']
        cityHTML['url'] = response.url
        cityHTML['pageid'] = response.meta['pageid']
        cityHTML['html'] = response.body.decode("gb18030").encode("UTF-8")
        yield cityHTML

        hotel_ids = response.selector.xpath('//div[@class="searchresult_list"]/@id').extract()
        cid = response.meta['cid']
        prefix = "http://hotels.ctrip.com/hotel/%s.html"
        for id in hotel_ids:
            tmp_url = prefix % id
            yield scrapy.Request(tmp_url, callback=self.parseHotelInfo, meta={'hotelid': id, 'cid': cid}) 
        
    def parseHotels(self, response):
        print "hotels url", response.url
        pages = response.selector.xpath('//input[contains(@class, "c_page_num")]/@data-pagecount').extract() 
        page_num = 0
        if len(pages) > 0:
            page_num = int(pages[0])
       # if page_num > 10:
        #    page_num = 10
        #deal page 1
        response.meta['pageid'] = 1
        cityHTML = items.CtripCityHTML()
        cityHTML['cid'] = response.meta['cid']
        cityHTML['url'] = response.url
        cityHTML['pageid'] = response.meta['pageid']
        cityHTML['html'] = response.body.decode("gb18030").encode("UTF-8")
        yield cityHTML

        hotel_ids = response.selector.xpath('//div[@class="searchresult_list"]/@id').extract()
        cid = response.meta['cid']
        prefix = "http://hotels.ctrip.com/hotel/%s.html"
        for id in hotel_ids:
            tmp_url = prefix % id
            yield scrapy.Request(tmp_url, callback=self.parseHotelInfo, meta={'hotelid': id, 'cid': cid}) 
            print "request hotl id ", tmp_url
        #deal page 2 to all
        cid = response.meta['cid']
        prefix = response.url
        for page in range(2, page_num+1):
        #for page in range(2, 2):
            tmpurl = prefix + "/p%s" % page
            yield scrapy.Request(tmpurl, callback=self.parseHotelsPage, meta={'cid': cid, 'pageid': page})

    def parse(self, response):
        if self.cities:
            pass
        else:
            cities = []
        
            index = response.body.find("suggestion=") + len("suggestion=")
	    cities = self.json_parse(response.body[index:])
            city_file = open('citylist.txt', 'wb')
            pickle.dump(cities, city_file)
            self.cities = cities

        hotel_prefix = "http://hotels.ctrip.com/hotel/"
        for city in self.cities:
            #code for debug and test
            yield city
            tmp_url = hotel_prefix + city['pinyin'] + city['cid']

            yield scrapy.Request(tmp_url, callback=self.parseHotels, meta={'cid': city['cid']})
